%!TEX root = /Users/dejean/Documents/ITU/2aar/MAIG-E2011/exam_project/report/simmar_report.tex
\section{Experiments}

For testing the stand-alone performance of the agents, we looked at the following three test scenarios:

\begin{itemize}
	\item 1000 games at difficulty 0 and 1, 500 at each difficulty
	\item 500 games at difficulty 0
	\item 500 games at difficulty 1
\end{itemize}

In each scenario, the same sequence of randomly generated seeds were used.

Finally, we gather statistics from benchmark agents in the Mario AI framework to compare against our agents.

For the experiment, we used a set of weights of our own choosing as the \emph{SystemOfValues} used to compute the fitness function. 

Most of these values are similar to what is found in the \emph{MarioCustomSystemOfValues} set by default, with the exception that the collection of coins and power-ups were changed to not contribute to fitness since none of the agents in the experiment contain any design intended to make them deliberately collect these. 

More emphasis was placed on winning the level, and also the speed at which the level is completed. Bonus points for kills using turtle shells is removed since again, it is not something that any of the agents in the test will ever be able to do deliberately at this point. 

The intent of these changes is to minimize the effect of coincidental opportunities on the performance score of the agents.

%%%%%%%%%%%% NEAT %%%%%%%%%%%%
\subsection{NEAT}

\begin{figure}[htp]
	\centerline{\includegraphics[width=0.9\columnwidth]{images/neat_performance.png}}
	\caption{NEAT agent performance, histograms}
	\label{neat_performance}
\end{figure}

\begin{table}
\begin{center}
\renewcommand{\arraystretch}{1.3}
\caption{\emph{NEATAgent} performance, statistics}
\label{neat_statistics}
\begin{tabular}{|c|c|c|c|c|}
\hline
& $\mu$ & $\tilde{x}$ & $\sigma$ & ${\sigma}^2$\\
\hline
\hline
Scenario 1 & 5677 & 4719 & 3156 & 9960000\\
Scenario 2 & 8526 & 9394 & 1689 & 2854200\\
Scenario 3 & 2837 & 2702 & 925 & 855420\\
\hline
\end{tabular}
\end{center}
\end{table}

Figure \ref{neat_performance} shows the distribution of fitness values for the three test scenarios, and table \ref{neat_statistics} shows the corresponding statistical measures.

%%%%%%%%%%%% XCS %%%%%%%%%%%%
\subsection{XCS}

\begin{figure}[htp]
	\centerline{\includegraphics[width=0.9\columnwidth]{images/xcs_performance.png}}
	\caption{XCS agent performance, histograms}
	\label{xcs_performance}
\end{figure}

\begin{table}
\begin{center}
\renewcommand{\arraystretch}{1.3}
\caption{XCSAgent performance, statistics}
\label{xcs_statistics}
\begin{tabular}{|c|c|c|c|c|}
\hline
& $\mu$ & $\tilde{x}$ & $\sigma$ & ${\sigma}^2$\\
\hline
\hline
Scenario 1 & 4191 & 3048 & 2651 & 7025100\\
Scenario 2 & 6167 & 5835 & 2413 & 5821800\\
Scenario 3 & 2225 & 2116 & 678 & 459010\\
\hline
\end{tabular}
\end{center}
\end{table}

Figure \ref{xcs_performance} shows the distribution of fitness values for the three test scenarios, and table \ref{xcs_statistics} shows the corresponding statistical measures.

%%%%%%%%%%%% COMPARISON %%%%%%%%%%%%
\subsection{Comparisons}
To evaluate the performances we ran the scenario 1 test on the two benchmark agents \emph{RandomAgent} and \emph{ForwardJumpingAgent}. The statistics in this scenario for all four agents are shown together in table \ref{comparisons}.

\begin{table}
\begin{center}
\renewcommand{\arraystretch}{1.3}
\caption{Comparisons with benchmark agents}
\label{comparisons}
\begin{tabular}{|l|c|c|c|c|}
\hline
& $\mu$ & $\tilde{x}$ & $\sigma$ & ${\sigma}^2$\\
\hline
\hline
\emph{NEATAgent} & 5677 & 4719 & 3156 & 9960000\\
\emph{XCSAgent} & 4191 & 3048 & 2651 & 7025100\\
\emph{RandomAgent} & 3384 & 2796 & 1792 & 3210800\\
\emph{ForwardJumpingAgent} & 5775 & 4967 & 2980 & 8882900\\
\hline
\end{tabular}
\end{center}
\end{table}